package com.xiaotu.spider.project.pageprocessor;

import java.util.ArrayList;
import java.util.List;
import java.util.Map;

import org.apache.commons.lang.StringUtils;
import org.springframework.context.annotation.Scope;
import org.springframework.stereotype.Component;

import com.xiaotu.common.util.Constants;
import com.xiaotu.common.util.FileUtil;
import com.xiaotu.common.util.HtmlUtil;
import com.xiaotu.common.util.KeywordEncode;
import com.xiaotu.common.util.PageUtils;
import com.xiaotu.common.util.SepratorUtil;
import com.xiaotu.spider.APageProcessor;

import net.sf.json.JSONArray;
import net.sf.json.JSONObject;
import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Request;

/**
 * @类名 TouTiaoPageProcessor
 * @日期 2017年5月22日
 * @作者 高海军
 * @功能
 */
@Component("TouTiaoPageProcessor")
@Scope("prototype")
public class TouTiaoPageProcessor extends APageProcessor
{
	private static final String CSV_LINE_SEPRATOR = "#***#";
	
	private String fileInput;
	
	private String fileOutput;
	
	@Override
	public void initRunParams(Map<String, Object> map)
	{
		this.fileInput = map.get("in") + StringUtils.EMPTY;
		this.fileOutput = map.get("out") + StringUtils.EMPTY;
	}
	
	@Override
	protected Request getStartRequest(JSONObject target)
	{
		return new Request(
				"http://www.toutiao.com/search_content/?offset=0&format=json&keyword="
						+ KeywordEncode.encode(target.getString("keyWord"))
						+ "&autoload=true&count=200&cur_tab=1");
	}
	
	@Override
	protected void setSiteRequest()
	{
		List<JSONObject> targetList = new ArrayList<JSONObject>();
		// for (JSONObject jsonObject : this.targetList)
		// {
		// for (String keyWorkd : jsonObject.getString("keyWords")
		// .split(SepratorUtil.SEP_COMMA_EN))
		// {
		// JSONObject target = new JSONObject();
		// target.put(Constants.TargetType.OTHER_TYPE_KEY,
		// Constants.TargetType.OTHER);
		// target.put("keyWord", keyWorkd);
		// targetList.add(target);
		// }
		//
		// }
		
		List<String> lineList = FileUtil.readFileByLine(this.fileInput);
		for (String line : lineList)
		{
			JSONObject target = new JSONObject();
			target.put(Constants.TargetType.OTHER_TYPE_KEY,
					Constants.TargetType.OTHER);
			target.put("keyWord", line.trim());
			targetList.add(target);
		}
		this.targetList = targetList;
		super.setSiteRequest();
	}
	
	@Override
	public void process(Page page)
	{
		if (page.getUrl().toString().contains("/search_content"))
			this.searchListProcess(page);
		else
			this.detailProcess(page);
	}
	
	private void detailProcess(Page page)
	{
		String content = page.getHtml().xpath("//div[@class='article-content']")
				.get();
		if (StringUtils.isEmpty(content))
			return;
		content = HtmlUtil.delHTMLTag(content);
		if (StringUtils.isEmpty(content))
			return;
		
		String time = page.getHtml().xpath("//span[@class='time']/text()")
				.get();
		String title = page.getHtml()
				.xpath("//h1[@class='article-title']/text()").get();
		String keyWord = page.getRequest().getExtra("keyWord") + "";
		
		FileUtil.writeFile(this.fileOutput,
				page.getUrl().toString() + CSV_LINE_SEPRATOR + keyWord
						+ CSV_LINE_SEPRATOR + title + CSV_LINE_SEPRATOR + time
						+ CSV_LINE_SEPRATOR + content
						+ SepratorUtil.LINE_SEPRATOR);
	}
	
	private void searchListProcess(Page page)
	{
		if (page.getJson() == null)
			return;
		JSONObject json = page.getJson().toObject(JSONObject.class);
		if (json == null || json.isEmpty() || !json.has("data"))
			return;
		JSONArray dataArr = json.getJSONArray("data");
		for (int i = 0; i < dataArr.size(); i++)
		{
			JSONObject oneObj = dataArr.getJSONObject(i);
			if (oneObj == null || oneObj.isEmpty()
					|| !oneObj.has("article_url"))
				continue;
			String url = oneObj.getString("article_url");
			if (!url.startsWith("http://toutiao.com"))
				continue;
			PageUtils.addTargetRequest(page, oneObj.getString("article_url"),
					page.getRequest().getExtras());
		}
	}
	
	@Override
	protected String getSpiderType()
	{
		return "TouTiao";
	}
	
	@Override
	public String getPageDataType(Page page)
	{
		return "TouTiao";
	}
	
}
